version: 0.1

android_test_host: amazon_linux_2

phases:
  install:
    commands:

  pre_test:
    commands:
      # Print this so that the upload script can read and process the benchmark config
      - echo "The benchmark config is {{ benchmark_config_id }}"

      # Download the model from S3
      - curl -s --fail '{{ model_path }}' -o model.zip
      - unzip model.zip && ls -la

      # Copy the model to sdcard. This prints too much progress info when the files
      # are large, so it's better to just silent them
      - adb -s $DEVICEFARM_DEVICE_UDID push *.bin /sdcard > /dev/null && echo OK
      - adb -s $DEVICEFARM_DEVICE_UDID push *.model /sdcard > /dev/null && echo OK
      - adb -s $DEVICEFARM_DEVICE_UDID push *.pte /sdcard > /dev/null && echo OK
      - adb -s $DEVICEFARM_DEVICE_UDID push *.json /sdcard > /dev/null && echo OK

      # Prepare the model and the tokenizer
      - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /sdcard/"
      - adb -s $DEVICEFARM_DEVICE_UDID shell "mkdir -p /data/local/tmp/minibench/"
      - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.bin /data/local/tmp/minibench/"
      - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.model /data/local/tmp/minibench/"
      - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.pte /data/local/tmp/minibench/"
      - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.json /data/local/tmp/minibench/"
      - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/minibench/*.bin"
      - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/minibench/*.model"
      - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/minibench/*.pte"
      - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/minibench/*.json"
      - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /data/local/tmp/minibench/"
      - adb -s $DEVICEFARM_DEVICE_UDID shell "run-as org.pytorch.minibench rm -rf files"

  test:
    commands:
      # Fail the test if the model doesn't exist, doing it here so that AWS can report the status back
      - echo "Verify model"
      - curl -I --fail '{{ model_path }}' || false

      # By default, the following ADB command is used by Device Farm to run your Instrumentation test.
      # Please refer to Android's documentation for more options on running instrumentation tests with adb:
      # https://developer.android.com/studio/test/command-line#run-tests-with-adb

      # Run the Instrumentation test for sanity check
      - echo "Starting the Instrumentation test"
      - |
        adb -s $DEVICEFARM_DEVICE_UDID shell "am instrument -r -w --no-window-animation \
        $DEVICEFARM_TEST_PACKAGE_NAME/$DEVICEFARM_TEST_PACKAGE_RUNNER 2>&1 || echo \": -1\"" |
        tee $DEVICEFARM_LOG_DIR/instrument.log

      # Parse the results
      - |-
        INSTRUMENT_LOG="$DEVICEFARM_LOG_DIR/instrument.log"

        DID_ANY_TESTS_START=$(grep "INSTRUMENTATION_STATUS_CODE: 1" $INSTRUMENT_LOG | wc -l);
        TESTS_PASSED=$(grep "INSTRUMENTATION_STATUS_CODE: 0" $INSTRUMENT_LOG | wc -l);
        TESTS_ERRORED=$(grep "INSTRUMENTATION_STATUS_CODE: -1" $INSTRUMENT_LOG | wc -l);
        TESTS_FAILED=$(grep "INSTRUMENTATION_STATUS_CODE: -2" $INSTRUMENT_LOG | wc -l);
        TESTS_IGNORED=$(grep "INSTRUMENTATION_STATUS_CODE: -3" $INSTRUMENT_LOG | wc -l);
        TESTS_ASSUMPTION_FAILED=$(grep "INSTRUMENTATION_STATUS_CODE: -4" $INSTRUMENT_LOG | wc -l);
        TESTS_PROCESSES_CRASHED=$(grep "INSTRUMENTATION_RESULT: shortMsg=Process crashed." $INSTRUMENT_LOG | wc -l);

      # And print the results so that the CI job can show them later
      - |
        INSTRUMENT_LOG="$DEVICEFARM_LOG_DIR/instrument.log"

        if [ $DID_ANY_TESTS_START -eq 0 ];
        then
          echo "[PyTorch] Marking the test suite as failed because no tests started!";
          false;
        elif [ $TESTS_FAILED -ne 0 ];
        then
          echo "[PyTorch] Marking the test suite as failed because it failed to load the model";
          false;
        elif [ $TESTS_ERRORED -ne 0 ];
        then
          echo "[PyTorch] Marking the test suite as failed because $TESTS_ERRORED tests errored!";
          false;
        elif [ $TESTS_PROCESSES_CRASHED -ne 0 ];
        then
          echo "[PyTorch] Marking the test suite as failed because the app crashed due to OOM!";
          false;
        # Check for this last to make sure that there is no failure
        elif [ $TESTS_PASSED -ne 0 ];
        then
          cat "${INSTRUMENT_LOG}"
        fi;

      # Run the new generic benchmark activity https://developer.android.com/tools/adb#am
      - echo "Determine model type"
      - |
        BIN_FOUND="$(adb -s $DEVICEFARM_DEVICE_UDID shell find /data/local/tmp/minibench/ -name '*.bin')"
        if [ -z "$BIN_FOUND" ]; then
          echo "No *.bin tokenizer files found in /data/local/tmp/minibench/"
        else
          echo "*.bin tokenizer files found in /data/local/tmp/minibench/"
        fi

        MODEL_FOUND="$(adb -s $DEVICEFARM_DEVICE_UDID shell find /data/local/tmp/minibench/ -name '*.model')"
        if [ -z "$MODEL_FOUND" ]; then
          echo "No *.model tokenizer files found in /data/local/tmp/minibench/"
        else
          echo "*.model tokenizer files found in /data/local/tmp/minibench/"
        fi

        JSON_FOUND="$(adb -s $DEVICEFARM_DEVICE_UDID shell find /data/local/tmp/minibench/ -name '*.json')"
        if [ -z "$JSON_FOUND" ]; then
          echo "No *.json tokenizer files found in /data/local/tmp/minibench/"
        else
          echo "*.json tokenizer files found in /data/local/tmp/minibench/"
        fi

      - echo "Collect device state before running"
      - |
        adb -s $DEVICEFARM_DEVICE_UDID shell 'cat /sys/devices/system/cpu/cpu*/cpufreq/stats/time_in_state /sys/devices/system/cpu/cpu*/cpufreq/stats/trans_table' > $DEVICEFARM_LOG_DIR/state_before.txt

      - echo "Run benchmark"
      - |
        adb -s $DEVICEFARM_DEVICE_UDID shell touch /data/local/tmp/result.etdump
        adb -s $DEVICEFARM_DEVICE_UDID shell am force-stop org.pytorch.minibench

        adb -s $DEVICEFARM_DEVICE_UDID shell dumpsys deviceidle force-idle
        adb -s $DEVICEFARM_DEVICE_UDID shell dumpsys deviceidle unforce
        adb -s $DEVICEFARM_DEVICE_UDID shell sleep 180

        if [ -n "$BIN_FOUND" ]; then
          adb -s $DEVICEFARM_DEVICE_UDID shell am start -W -n org.pytorch.minibench/.BenchmarkActivity \
            --es "model_dir" "/data/local/tmp/minibench" \
            --es "tokenizer_path" "/data/local/tmp/minibench/tokenizer.bin" \
            --ei "num_iter" 5 --ei "num_warm_up_iter" 2
        elif [ -n "$MODEL_FOUND" ]; then
          adb -s $DEVICEFARM_DEVICE_UDID shell am start -W -n org.pytorch.minibench/.BenchmarkActivity \
            --es "model_dir" "/data/local/tmp/minibench" \
            --es "tokenizer_path" "/data/local/tmp/minibench/tokenizer.model" \
            --ei "num_iter" 5 --ei "num_warm_up_iter" 2
        elif [ -n "$JSON_FOUND" ]; then
          adb -s $DEVICEFARM_DEVICE_UDID shell am start -W -n org.pytorch.minibench/.BenchmarkActivity \
            --es "model_dir" "/data/local/tmp/minibench" \
            --es "tokenizer_path" "/data/local/tmp/minibench/tokenizer.json" \
            --ei "num_iter" 5 --ei "num_warm_up_iter" 2
        else
          adb -s $DEVICEFARM_DEVICE_UDID shell am start -W -n org.pytorch.minibench/.BenchmarkActivity \
            --es "model_dir" "/data/local/tmp/minibench"
        fi

      - echo "Collect device state after running"
      - |
        adb -s $DEVICEFARM_DEVICE_UDID shell 'cat /sys/devices/system/cpu/cpu*/cpufreq/stats/time_in_state /sys/devices/system/cpu/cpu*/cpufreq/stats/trans_table' > $DEVICEFARM_LOG_DIR/state_after.txt

  post_test:
    commands:
      - echo "Gather benchmark results"
      - |
        BENCHMARK_RESULTS=$(adb -s $DEVICEFARM_DEVICE_UDID shell run-as org.pytorch.minibench cat files/benchmark_results.json)
        ATTEMPT=0
        MAX_ATTEMPT=10
        while [ -z "${BENCHMARK_RESULTS}" ] && [ $ATTEMPT -lt $MAX_ATTEMPT ]; do
          echo "Waiting for benchmark results..."
          BENCHMARK_RESULTS=$(adb -s $DEVICEFARM_DEVICE_UDID shell run-as org.pytorch.minibench cat files/benchmark_results.json)
          sleep 30
          ((ATTEMPT++))
        done

        adb -s $DEVICEFARM_DEVICE_UDID shell run-as org.pytorch.minibench ls -la files/
        # Trying to pull the file using adb ends up with permission error, but this works too, so why not
        echo "${BENCHMARK_RESULTS}" > $DEVICEFARM_LOG_DIR/benchmark_results.json

        adb -s $DEVICEFARM_DEVICE_UDID pull /data/local/tmp/result.etdump $DEVICEFARM_LOG_DIR/result.etdump

artifacts:
  # By default, Device Farm will collect your artifacts from the $DEVICEFARM_LOG_DIR directory.
  - $DEVICEFARM_LOG_DIR
